package com.sunzm.spark.core

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object RDDdependencies {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("RDD转换类算子示例")
      .setMaster("local[*]")
    val sc: SparkContext = new SparkContext(conf)

    val fileRDD: RDD[String] = sc.textFile("data/spark/rdd/word.txt")
    println(fileRDD.dependencies.toBuffer)
    println("----------------------")

    val wordRDD: RDD[String] = fileRDD.flatMap(_.split(","))
    println(wordRDD.dependencies.toBuffer)
    println("----------------------")

    val mapRDD: RDD[(String, Int)] = wordRDD.map((_,1))
    println(mapRDD.dependencies.toBuffer)
    println("----------------------")

    val resultRDD: RDD[(String, Int)] = mapRDD.reduceByKey(_+_)
    println(resultRDD.dependencies.toBuffer)

    resultRDD.collect()

    sc.stop()
  }
}
